import sys
from bs4 import BeautifulSoup
import requests
import json
import jsonpath


# 爬虫地址
url='https://www.digikey.cn/zh/products'
headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36 Edg/105.0.1343.33'
}

# url='https://www.baidu.com/'
res = requests.get(url,headers=headers)
result = res.content.decode('utf-8')

file = open('C:\\Users\\Administrator\\Desktop\\test\\temp\\ceshi.json','w',encoding='utf-8')
json.dump(result,file,ensure_ascii=False)
tree = BeautifulSoup(result, 'html.parser')
jsonStr = tree.find('script',id='__NEXT_DATA__').get_text()

# print(jsonStr)
jsonObj = json.loads(jsonStr)
productslist = jsonpath.jsonpath(jsonObj,'$..categories')
length = productslist.count
index = 0
subfile = open('C:\\Users\\Administrator\\Desktop\\test\\temp\\categories.txt','w',encoding='utf-8')
for pitem in productslist[0]:
    subfile.write('------------------------------------------------------------\n')
    subfile.writelines(pitem['label']+'\n')

    subitems = jsonpath.jsonpath(pitem,'$..subCategories')
    for subitem in subitems[0]:
        subItemUrl = subitem['url']
        subItemId = subItemUrl[subItemUrl.rindex('/')+1:]
        pageUrl= 'https://www.digikey.cn/products/api/v4/filter-page/'+subItemId
        cookieStr='utag_multi=koddi_guid:6d46ee3b-758f-413b-983b-51f45aeb0a78;exp-session; _ga=GA1.2.97448219.1663317520; _cs_c=0; _gcl_au=1.1.1587973258.1663317521; ai_user=mt5xvSKBtzhCuXdiQAD/Ti|2022-09-16T08:38:39.895Z; _evga_8292=14e6e943d1d21444.; dkuhint=false;  TS01d90084=01460246b6c3d7a2e58b6ad2ef2cbcc4964e1ce6da1af3df1d89ed8d0d21e15e1147fe424737b52c423c4517ded523b10350fe4467; _cs_cvars={{"1":["Page Title","Part Search - Family Page"],"2":["Page Site","CN"],"3":["Page Type","PS"],"4":["Page Sub Type","FAM"],"5":["Page Content Group","Part Search"],"6":["PageContentSubGroup","Category Page"],"7":["Page ID","{}"],"8":["Page Language","zh"],"15":["Page State","Sort Order Test - Default"],"16":["L1 cat","{}"]}}; TScb802422027=089265d43dab20003805eef5b68503cf0def5511abab7949ee4fb18d1250c556336e85af3e1ba4c508e8d6fdda113000ec8f8b39110ec2a272823cf2a120bf521edc14a03e419f8a20de99f27985283a5d0adbdc24fa297f19d70f46f1220bb2; search={{"usage":{{"dailyCount":3,"lastRequest":"2022-09-19T01:27:39.203Z"}},"version":1}}; TS01173021=01a0cfa1f24ffa67a6e0d24ba9c290395034fa65aacb575ac917e97306f28e881444a95408f5f2fcc95528f1d44787bf05f7e1c4d2; TSf2b291e0027=085f3c8666ab2000a005f3852bc3e0bdf8def787337568f8f83f57d4f367efa9849a9ff6d98aadcc087313ea1b1130003a47fc34ef18982d7a8062e009162fc9d3b82d3f8fe3fbd95f485a6c7bb3b8024e49979883d081844d759452ed1d9435; _abck=BBF74C5CBB6689672534151C925176C6~0~YAAQFQ1x32MmZT6DAQAA665bUwhOQppPLWyjNl+Fj/73Ga7vSWo3dZ6Q7RQx3NNCa4En7UaiabYMmlCJbaNgL8crflgzROMuw38ye2X32TMFfUxfbRV+8TQmnMkKXWJgNeroLP8XsQ3aGxsdZchDYh/k1yj+df5LCmdMXqyoFtDSes1kLlYI7kyoNvaPMu29KCa02c+Xy7cq0InY50M6dXcyAXpnsNxoSWEUwM5pLzMNI/MtpZk/hUndM0MrfRLukjKJTT/VMmbDPgHU3DQuF+jpK3bG0lc4mOr+yUZoiZJW7UJEdtSFO8w00OOaPG7JhLQXRd6QXMKRoOdKunLDugxRKY8fstthkPphF3+/GFaR+hfZi7c5McnAX21lhDUDq2VREj41LAHz2iELUkfXqsAuoKgN~-1~-1~-1; bm_sz=FE91917D556BB8AA86416AFF111F09B1~YAAQFQ1x32YmZT6DAQAA665bUxEbon+iHNWaCZW1AdrUTGaekgPQGCI9qskVxX67Df1SK9sjpuvKyF71n8VRWXi/c6jcHnu9CJZab8q95EgHi4Py+7Vh8KiEmUj61aSY8/1F0axuBsJUWqQr5b8dW757sKo9cVUbv7fh89P0StIQQ43wNL/ooVGD2LvqE/O14uMbhi59MD5QuuS65qixpdD+jgbkjHvzHWvCwQ9TW0n0s8fD1EjdT0RO4hGvxP+HJlGsuJnE/FWBIotFjhowfSq006kemjG1+e37nvQAkynaTwM=~3162933~3753014; RT="z=1&dm=digikey.cn&si=l1rx44aqw6m&ss=l85mkvtl&sl=0&tt=0"; TS01c02def=01460246b6af63da77482a0bc2cef6195b19e04429c2c4884a1f6c7d8f1462628625a572ddc5ecb94de7f9ee2609aa7c907ca9cd3d; TS01b6e805=01460246b6af63da77482a0bc2cef6195b19e04429c2c4884a1f6c7d8f1462628625a572ddc5ecb94de7f9ee2609aa7c907ca9cd3d; TS809e22c5027=08374f23c1ab20006a262ea253843fc95d58defb293386adf046ede3409d87672058e7c236b7f7de086d45a317113000c2208d7f3c23c7e467548374d7887ce2fe07b5d29566080f8ec2092ee0cac3a434e648c2f70f3a2dc89474870930a8f7; TS01cba742=01460246b639fccfccb83a479f94cdd5dedbab9af13438127f4986ecd40667ed1bf06380d3458d1f590a81ee17bdc48ac6ae46e477; TS01bd62a8=01460246b639fccfccb83a479f94cdd5dedbab9af13438127f4986ecd40667ed1bf06380d3458d1f590a81ee17bdc48ac6ae46e477; TS605a4192027=08374f23c1ab2000f1ddadd7fd4eaa6b5636744179fdcacb53a7432e38c64feea777fb664077acfe0856297e131130002d4354de3e1a308667548374d7887ce2eeda41a9474ef3878b25af781d52a437b597d72f9dd1463416b1b053f78caae8; TS016ca11c=01460246b6b858356ec3433672ef51252243bcca14f062d7248aa08dea91600c4f831d8b8050023cd6732a273b442495b26a15c126; TS011a64f6=01460246b6b858356ec3433672ef51252243bcca14f062d7248aa08dea91600c4f831d8b8050023cd6732a273b442495b26a15c126; TSc580adf4027=08374f23c1ab2000943ab038ae3b5ac58e2f197c7566479c6fd4479bbabe34556f82242227573a5d0890e6e9aa1130003bfe7cdcfe14987c67548374d7887ce2572a1c38370446954cccdc46428534065e1700990728b5dd1c4dcf0507a5100e; utag_main=v_id:01834572dc0800865ebbac8f4ed005081001707900bd0$_sn:7$_se:1$_ss:1$_st:1663552663205$ses_id:1663550863205;exp-session$_pn:1;exp-session; _cs_mk=0.3053187413364573_1663550863275; _gid=GA1.2.1327268672.1663550864; website#lang=; TS0198bc0d=01460246b666cc3ffa0ae384344edecd4268fff8e97c51eda31d85a22966f1e77e5a8279e8c843cc7ddcd9ea72f5a1246458785f5c; TSe14c7dc7027=08374f23c1ab20006ecd770fa156f5d02ab64119b52921a39e13406016f676ddd1a5fad502ccf6b908ab20d6c7113000ee73c2893e622c1dd29386eab42f2ad6727f5bb013d241f42dfa16b78b6a1dc8fddbaaa2a5a83ae52eba49028792244b; TSac889e6d027=08374f23c1ab200007360a1b0addb651f00e429a163b0f5fc954f07522c7106be1c23df0795e2cb608d7efb64c11300008d5b921284da7f7d29386eab42f2ad685acab94487255e71dccd33f28d1745bb92770c2f138eba1cb69fab60dc56293; TS01afc56e=01460246b624c7a793b77de59d345e5b95821df446ae42361120c83527b290e67eace810c67f6063c09d68e9dd5051803df6050101; TSf44f2996027=08374f23c1ab2000f5e088133570cf51a76c973c135e7e16eb29fdf86c70f73890f2786096b669c308a00d2345113000eba2b63305e144f5d29386eab42f2ad69e3ab73897d5285358157e5fee514b1c02e74da11a359fb8712a159f8c189964; _cs_id=4cbbd654-698c-a4a7-9ac2-52ba1ae9e129.1663317520.6.1663550863.1663550324.1.1697481520613; _cs_s=3.0.0.1663552663967; bm_mi=1DAAA956FB6D41868D93C0ACE118FA19~YAAQFQ1x37swZT6DAQAAMNNbUxEps8DR+9eM4vpyl/KY8OKa8FRwMN9GnN1R8vJ2E9G6VWP43oF6JSW94UnefsiQviibTuukkgBSawdfKxdxxR0ECjwGRQUp2R4aTloO4TVHC2V4n6CjFir/eUBO2eoAeggIYxG2BDi+GjjPb+DX6I2kjc4YMlOKK2IJnOuw9MDhY5I6IFQAvTB67emcXCYvDsZuBHVZh258IganQq/Quo0OTmHO5G9JZI4ADCCFoy9uGcPugHWytq35FckkGthQq1JSKhqLiTj2O/uV+GD2q6tuwv3oP48g+BdjLhxQgExIV/eA6taXVB0ioa9AO2Sj0gMQfIA99y5/Zn92t/m/SKxsbLkFN2puhS0yf+iAFid/rC+B6Q==~1; bm_sv=6EB21C0DC3256CE317EA6B598D86D996~YAAQFQ1x34UzZT6DAQAA1dxbUxESfP73KIwhwaGcxZTwyEkDIYmb5GQ8rkLjtxeM7994tVnrM0dNwm94NYXGdJ845Vq5FHij0CwppKMF6j4cmRFVo9AqtjB5sJyTo/QpSXwwqasr3ekY9q+Z16Jg2papJ0RpSTO68eFCwmaeevRmO4ocP7ZtU4SpU8Ii2vM1baqyCxxutmanwBD7ItOd37/o6v6SXAHwjOzTQFucnpQ+Fxd/iTDxqiv2MIF97pyQ~1; ai_session=QikWskMO5TlgXWXynx3CAj|1663549953072|1663550925554; ak_bmsc=C144990509E1935B8F27577A92D15961~000000000000000000000000000000~YAAQFQ1x35maZT6DAQAA6zRdUxFkR9FvHwvt7MWlhnTpqRys5ulbrUFoFksOtj4lGOtAVlSEngbn4uWewEhZVZKg96XeLqZuj6elOtye/WvaNz/BlrKc4eJmK+PB5D7xfuAG3yd+lipWWgHSSdamC6Y3FdWyE7BTDeO4vMWRIkVfVe4OmzIaOwv69jXJsjo7ioT2uJWqyH+eEVbs1bgA4ouozQ56rtlatizYtWqhYvIsTi+RiN3d+GRLOO5wB/OMTFvxaGivT95mdvj2NBTcfWrsegW3aFsqDTVb/fXhDUkQNnPYhk9TIMFhiC8CT2DzvGPkK4lKg1fYngLRcqBHlmk7R+aoxdTXNaxS/zNz0vTGnjoS+ue2iypiZqzkfNWkqvB6ToOkpDfCo/RSDDrPLx2pd+SD7rs9ctANKtFS+SbbH1Gx0hczuY3zM/B8FV1Z2ER1UxDNzOIpCb2pBHVsk2vTTAmLTZ3ZwSXuVF0bCzskpg8QGrzAdlRNAwtD9suo5TSJvP6YngRBbQlL3LD9Bz7CRaekPUvXg9utnYhNB84E1YOtGlrZ3q9AkRLMN1V/EfORGjI7AnC8; utm_data_x=html_element1=btn-page-{},html_element2=pagination-container,html_element3=MuiGrid-root,html_element4=MuiGrid-root MuiGrid-container,undefined=jss182,ref_page_event=Change Page in Search Results,ref_page_type=PS,ref_page_sub_type=FAM,ref_page_id={},ccookie=2022-09-16T08:38:40.165Z,ref_page_state=Sort Order Test - Default,ref_pers_state={{"search2":"modern","PLS":"Scrolling"}},ref_part_search_term=,ref_part_search_term_ext=,ExtRun=428.1|428.5|435.22|409.2,ExtRun=428.1|428.5|435.22|409.2|429.2|428.1; _dd_s=rum=0&expire=1663551868397'
        detailheaders = {
            'accept-language': 'zh-cn',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36 Edg/105.0.1343.33',
            'cookie':cookieStr.format(subItemId,pitem['label'],'2',subItemId).encode("utf-8").decode("latin1"),
            'x-currency': 'CNY'
        }
        if index < 1:
            pageReq = requests.get(pageUrl,headers=detailheaders)
            pageResult = pageReq.content.decode('utf-8')
            subfile.writelines(pageResult)
            index += 1

        subfile.writelines('type={};url={};pageUrl={}\n'.format(subitem['label'],subitem['url'],pageUrl))

# Json测试
# jsonStr = '{"city":"中文","name":"中文"}';
# r = json.loads(jsonStr);
# print(r);